package com.chance.cc.crawler.development.scripts.hao3490;

import com.chance.cc.crawler.core.CrawlerEnum;
import com.chance.cc.crawler.core.CrawlerRecordContext;
import com.chance.cc.crawler.core.downloader.HttpPage;
import com.chance.cc.crawler.core.pipeline.result.CrawlerData;
import com.chance.cc.crawler.core.record.CrawlerRequestRecord;
import com.chance.cc.crawler.core.script.crawler.CrawlerCommonScript;
import com.chance.cc.crawler.core.selector.Selectable;
import com.chance.cc.crawler.core.tags.crawler.CrawlerResultTags;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.DateUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.text.ParseException;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import static com.chance.cc.crawler.core.CrawlerEnum.CrawlerDataType.article;
import static com.chance.cc.crawler.core.CrawlerEnum.CrawlerDataType.interaction;

public class Hao3490CrawlerScript extends CrawlerCommonScript {
    private Logger log = LoggerFactory.getLogger(Hao3490CrawlerScript.class);

    private static final String entranceUrl = "http://www.3490.cn/news/";
    private static final String moduleUrl = "http://www.3490.cn/[a-zA-Z\\/]*_\\d+.html";
    private static final String articleUrl = "http://\\S*.3490.cn/[a-zA-Z\\-\\/]*\\d+.html";


    @Override
    public String domain() {
        return "3490";
    }

    @Override
    public boolean crawlerCheck(CrawlerRequestRecord crawlerRequestRecord) {
        return true;
    }

    @Override
    public void initUrlRegulars() {
        addUrlRegular("http://www.3490.cn/");
        addUrlRegular(moduleUrl);
        addUrlRegular(articleUrl);
    }

    @Override
    public List<CrawlerRequestRecord> parseLinks(CrawlerRequestRecord crawlerRequestRecord, HttpPage httpPage) {
        List<CrawlerRequestRecord> parsedLinks = new ArrayList<>();
        String requestUrl = httpPage.getRequest().getUrl();
        if (requestUrl.matches("http://www.3490.cn/")) {
            List<String> pageList = httpPage.getHtml().xpath("//div[@class=\"main-nav\"]/ul/li/a/@href").all();
            for (String listUrl : pageList) {
                if (listUrl.matches("http://www.3490.cn/(bns|brand)/")) {
                    CrawlerRequestRecord turnPageRequest = CrawlerRequestRecord.builder()
                            .turnPageRequest(crawlerRequestRecord)
                            .recordKey(listUrl + "news/index_1.html")
                            .httpUrl(listUrl + "news/index_1.html")
                            .releaseTime(System.currentTimeMillis())
                            .notFilterRecord()
                            .build();

                    parsedLinks.add(turnPageRequest);
                } else if (listUrl.matches("http://www.3490.cn/news/")) {
                    CrawlerRequestRecord turnPageRequest = CrawlerRequestRecord.builder()
                            .turnPageRequest(crawlerRequestRecord)
                            .recordKey(listUrl + "tuijian_1.html")
                            .httpUrl(listUrl + "tuijian_1.html")
                            .releaseTime(System.currentTimeMillis())
                            .notFilterRecord()
                            .build();
                    parsedLinks.add(turnPageRequest);
                }
            }
        } else if (requestUrl.matches(moduleUrl)) {
            String site = requestUrl.substring(requestUrl.lastIndexOf("/") + 1, requestUrl.lastIndexOf("_"));
            //下一页地址的解析
            List<Selectable> nodes1 = httpPage.getHtml().xpath("//div[@class=\"num-list\"]/a|//div[@class=\"fr\"]/a").nodes();
            String nextPageUrl = "";
            for (Selectable selectable : nodes1) {
                String text = selectable.xpath("./text()").get();
                if ("下页".equals(text)) {
                    nextPageUrl = selectable.xpath("./@href").get();
                    break;
                }
            }

            if (StringUtils.isNotEmpty(nextPageUrl)) {
                nextPageUrl = requestUrl.substring(0, requestUrl.lastIndexOf("/") + 1) + nextPageUrl;
                CrawlerRequestRecord turnPageRequest = CrawlerRequestRecord.builder()
                        .turnPageRequest(crawlerRequestRecord)
                        .recordKey(nextPageUrl)
                        .httpUrl(nextPageUrl)
                        .releaseTime(System.currentTimeMillis())
                        .build();

                parsedLinks.add(turnPageRequest);
            }


            List<Selectable> nodes = httpPage.getHtml().xpath("//div[@class=\"news_list\"]/ul/li|//div[@class=\"list-left\"]/ul/li").nodes();
            for (Selectable node : nodes) {
                String itemUrl = node.xpath("./a/@href").get();
                if (StringUtils.isBlank(itemUrl)) {
                    continue;
                }

                if (site.matches("tuijian")) {
                    itemUrl = itemUrl.matches(articleUrl) ? itemUrl : requestUrl.substring(0, requestUrl.lastIndexOf("/") + 1) + itemUrl;
                } else {
                    itemUrl = itemUrl.matches(articleUrl) ? itemUrl : "http://www.3490.cn" + itemUrl;
                }

                String pubTime = node.xpath("./div//span[@class=\"date\"]").get();
                if (StringUtils.isBlank(pubTime)) {
                    continue;
                }

                try {
                    long releaseTime = DateUtils.parseDate(pubTime, "yyyy/MM/dd").getTime();

                    CrawlerRequestRecord itemRecord = CrawlerRequestRecord.builder()
                            .itemPageRequest(crawlerRequestRecord)
                            .recordKey(itemUrl)
                            .httpUrl(itemUrl)
                            .needParsed(false)
                            .releaseTime(releaseTime)
                            .resultLabelTag(article)
                            .resultLabelTag(interaction)
                            .build();
                    parsedLinks.add(itemRecord);
                } catch (ParseException e) {
                    e.printStackTrace();
                }

            }
        }

        return parsedLinks;
    }

    @Override
    public List<CrawlerData> washPage(CrawlerRequestRecord crawlerRecord, HttpPage page) {
        List<CrawlerData> crawlerDataList = new ArrayList<>();
        CrawlerResultTags crawlerResultTags = crawlerRecord.tagsCreator().resultTags();
        String requestUrl = page.getRequest().getUrl();

        if (crawlerResultTags.hasDataType(article)) {
            crawlerDataList.add(washArticle(crawlerRecord, page));

        }

        if (crawlerResultTags.hasDataType(interaction)) {
            crawlerDataList.add(washInteraction(crawlerRecord, page));
        }

        return crawlerDataList;
    }

    public CrawlerData washArticle(CrawlerRequestRecord crawlerRequestRecord, HttpPage httpPage) {
        String itemUrl = httpPage.getRequest().getUrl();
        try {
            String title = httpPage.getHtml().xpath("//div[@class=\"news_hd\"]/h1/text()|//div[@id=\"cp_xwXiangxcont_t\"]/h1/text()|//div[@class=\"article_hd\"]/h1").get();
            String source = httpPage.getHtml().xpath("//div[@class=\"tq\"]/span/a/text()|//div[@id=\"cp_xwXiangxcont_tq\"]/span/a/text()").get();
            List<String> all = httpPage.getHtml().xpath("//div[@class=\"tq\"]/text()|//div[@id=\"cp_xwXiangxcont_tq\"]/text()|//div[@class=\"sub_hd\"]/text()").all();
            String releaseTime = "";
            String author = "";
            for (String data : all) {
                String m = StringUtils.isNotEmpty(data) ? getString("\\d{4}[\\/\\-]*\\d+[\\/\\-]*\\d+[\\d\\:\\ ]*",data): "";
                if(StringUtils.isNotEmpty(m)){
                    releaseTime = m;
                    author = data.substring(data.lastIndexOf("By") + 2,data.lastIndexOf("阅读"));
                    break;
                }
            }
            String parsePatterns = "";
            if(releaseTime.matches("\\d{4}/\\d+/\\d+ \\d+:\\d+:\\d+")){
                parsePatterns = "yyyy/MM/dd HH:mm:ss";
            }else if(releaseTime.matches("\\d{4}/\\d+/\\d+")){
                parsePatterns = "yyyy/MM/dd";
            }else if(releaseTime.matches("\\d{4}-\\d+-\\d+ \\d+:\\d+:\\d+")){
                parsePatterns = "yyyy-MM-dd HH:mm:ss";
            }

            List<String> texts = httpPage.getHtml().xpath("//div[@class=\"news_body\"]/p//text()|//div[@id=\"cp_xwXiangxcont_m\"]/p//text()|//div[@class=\"text_body\"]/p//text()").all();
            StringBuffer conents = new StringBuffer();
            for (String text : texts) {
                conents.append(text).append(" ");
            }

            String articleKey = "";
            if(itemUrl.matches("\\S*-\\d+.html")){
                articleKey = itemUrl.substring(itemUrl.lastIndexOf("-") + 1,itemUrl.lastIndexOf("."));
            }else if(itemUrl.matches("\\S*/\\d+.html")){
                articleKey = itemUrl.substring(itemUrl.lastIndexOf("/") + 1,itemUrl.lastIndexOf("."));
            }

            CrawlerData crawlerData = CrawlerData.builder()
                    .data(crawlerRequestRecord, httpPage)
                    .dataId(StringUtils.joinWith("-", crawlerRequestRecord.getDomain(), CrawlerEnum.CrawlerDataType.article.enumVal(), articleKey))
                    .resultLabelTag(CrawlerEnum.CrawlerDataType.valueOf("article"))
                    .url(itemUrl)
                    .releaseTime(DateUtils.parseDate(releaseTime, parsePatterns).getTime())
                    .addContentKV("content", conents.toString().trim())
                    .addContentKV("title", title)
                    .addContentKV("source",source)
                    .addContentKV("author", author)
                    .build();
            return crawlerData;
        } catch (ParseException e) {
            log.error(e.getMessage(), e);
        }
        return null;

    }

    public CrawlerData washInteraction(CrawlerRequestRecord crawlerRequestRecord, HttpPage httpPage) {
        String itemUrl = httpPage.getRequest().getUrl();

        String views = getString("阅读\\(\\d+\\)", httpPage.getRawText());
        views = StringUtils.isNotEmpty(views)?views.substring(views.lastIndexOf("(") + 1,views.lastIndexOf(")")) : "0";

        List<String> all = httpPage.getHtml().xpath("//div[@class=\"tq\"]/text()|//div[@id=\"cp_xwXiangxcont_tq\"]/text()|//div[@class=\"sub_hd\"]/text()").all();
        String releaseTime = "";
        for (String data : all) {
            data = StringUtils.isNotEmpty(data) ? getString("\\d{4}[\\/\\-]*\\d+[\\/\\-]*\\d+[\\d\\:\\ ]*",data): "";
            if(StringUtils.isNotEmpty(data)){
                releaseTime = data;
                break;
            }
        }
        String parsePatterns = "";
        if(releaseTime.matches("\\d{4}/\\d+/\\d+ \\d+:\\d+:\\d+")){
            parsePatterns = "yyyy/MM/dd HH:mm:ss";
        }else if(releaseTime.matches("\\d{4}/\\d+/\\d+")){
            parsePatterns = "yyyy/MM/dd";
        }else if(releaseTime.matches("\\d{4}-\\d+-\\d+ \\d+:\\d+:\\d+")){
            parsePatterns = "yyyy-MM-dd HH:mm:ss";
        }

        String articleKey = "";
        if(itemUrl.matches("\\S*-\\d+.html")){
            articleKey = itemUrl.substring(itemUrl.lastIndexOf("-") + 1,itemUrl.lastIndexOf("."));
        }else if(itemUrl.matches("\\S*/\\d+.html")){
            articleKey = itemUrl.substring(itemUrl.lastIndexOf("/") + 1,itemUrl.lastIndexOf("."));
        }

        CrawlerData crawlerData = null;
        try {
            crawlerData = CrawlerData.builder()
                    .data(crawlerRequestRecord, httpPage)
                    .parentId(StringUtils.joinWith("-", crawlerRequestRecord.getDomain(), article.enumVal(), articleKey))
                    .dataId(StringUtils.joinWith("-", crawlerRequestRecord.getDomain(), interaction.enumVal(), articleKey))
                    .resultLabelTag(CrawlerEnum.CrawlerDataType.valueOf("interaction"))
                    .url(itemUrl)
                    .releaseTime(DateUtils.parseDate(releaseTime,parsePatterns).getTime())
                    .addContentKV("views", views)
                    .build();
        } catch (ParseException e) {
            e.printStackTrace();
        }
        return crawlerData;
    }

    @Override
    public void afterExecute(CrawlerRecordContext crawlerRecordContext) {

    }

    private static String getString(String regx, String input) {
        Pattern compile = Pattern.compile(regx);
        Matcher matcher = compile.matcher(input);
        List<String> urlList = new ArrayList<>();
        while (matcher.find()) {
            return matcher.group(0);
        }
        return null;
    }
}
